this_notebook_name = "BreastSegmentationStudy-TF2"

# Update this folder name for your computer

local_data_folder = "/Usersß/Josh Ehrlich/Courses/CISC881/Project/data"
overwrite_existing_data_files = False

# All results and output will be archived with this timestamp

import datetime
save_timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
print("Save timestamp: {}".format(save_timestamp))

# Learning parameters

import numpy as np

ultrasound_size = 128
num_classes = 2
num_epochs = 200
batch_size = 64
max_learning_rate = 0.02
min_learning_rate = 0.000001
regularization_rate = 0.0001
filter_multiplier = 15
class_weights = np.array([0.15, 0.85])
learning_rate_decay = (max_learning_rate - min_learning_rate) / num_epochs

# Training data augmentation parameters

max_shift_factor = 0.12
max_rotation_angle = 10
max_zoom_factor = 1.1
min_zoom_factor = 0.8

# Evaluation parameters

acceptable_margin_mm = 1.0
mm_per_pixel = 1.0

roc_thresholds = [0.9, 0.8, 0.7, 0.65, 0.6, 0.55, 0.5, 0.45, 0.4, 0.35, 0.3, 0.25, 0.2, 0.15, 0.1,
                  0.08, 0.06, 0.04, 0.02, 0.01,
                  0.008, 0.006, 0.004, 0.002, 0.001]

'''
Provide NxM numpy array to schedule cross validation
N rounds of validation will be performed, leaving out M patients in each for validation data
All values should be valid patient IDs, or negative. Negative values are ignored.

Example 1: a leave-one-out cross validation with 3 patients would look like this:
validation_schedule_patient = np.array([[0],[1],[2]])

Example 2: a leave-two-out cross validation on 10 patients would look like this:
validation_schedule_patient = np.array([[0,1],[2,3],[4,5],[6,7],[8,9]])

Example 3: leave-one-out cross validation with 3 patients, then training on all available data (no validation):
validation_schedule_patient = np.array([[0],[1],[2],[-1]])
'''
validation_schedule_patient = np.array([[0,1,2,3], [7,11,13,18], [9,14,21,31], [19,22,29,32], [20,23,27,30]])

# Uncomment for faster debugging

roc_thresholds = [0.8, 0.6, 0.4, 0.2, 0.1, 0.01, 0.001]
num_epochs = 200

Save timestamp: 2020-11-25_15-52-21

import os
from random import sample
from pathlib import Path

from ipywidgets import IntProgress
from IPython.display import display, HTML

import diskcache
import girder_client
import matplotlib.pyplot as plt
import pandas as pd

import tensorflow as tf

import ultrasound_batch_generator as generator
import evaluation_metrics

from girder_apikey import girder_apikey

# Import aigt modules

import sys
parent_folder = os.path.dirname(os.path.abspath(os.curdir))
sys.path.append(parent_folder)

import Models.segmentation_unet as unet
import utils

# Creating standard folders to save data and logs

data_arrays_fullpath, notebooks_save_fullpath, results_save_fullpath, models_save_fullpath, val_data_fullpath =\
    utils.create_standard_project_folders(local_data_folder)

# Fetching Girder data

girder_url = "https://pocus.cs.queensu.ca/api/v1"
data_csv_file = "BreastGirder.csv"
girder_key = girder_apikey

ultrasound_arrays_by_patients, segmentation_arrays_by_patients =\
    utils.load_girder_data(data_csv_file, data_arrays_fullpath, girder_url, girder_key)
    
n_patients = len(ultrasound_arrays_by_patients)

for i in range(n_patients):
    print("Patient {} has {} ultrasounds and {} segmentations".format(
        i, ultrasound_arrays_by_patients[i].shape[0], segmentation_arrays_by_patients[i].shape[0]))

Patient 0 has 379 ultrasounds and 379 segmentations
Patient 1 has 292 ultrasounds and 292 segmentations
Patient 2 has 426 ultrasounds and 426 segmentations
Patient 3 has 418 ultrasounds and 418 segmentations
Patient 4 has 442 ultrasounds and 442 segmentations
Patient 5 has 224 ultrasounds and 224 segmentations
Patient 6 has 272 ultrasounds and 272 segmentations
Patient 7 has 288 ultrasounds and 288 segmentations
Patient 8 has 175 ultrasounds and 175 segmentations
Patient 9 has 113 ultrasounds and 113 segmentations
Patient 10 has 237 ultrasounds and 237 segmentations
Patient 11 has 180 ultrasounds and 180 segmentations
Patient 12 has 5 ultrasounds and 5 segmentations
Patient 13 has 402 ultrasounds and 402 segmentations
Patient 14 has 295 ultrasounds and 295 segmentations
Patient 15 has 331 ultrasounds and 331 segmentations
Patient 16 has 145 ultrasounds and 145 segmentations
Patient 17 has 253 ultrasounds and 253 segmentations
Patient 18 has 158 ultrasounds and 158 segmentations
Patient 19 has 187 ultrasounds and 187 segmentations
Patient 20 has 135 ultrasounds and 135 segmentations
Patient 21 has 107 ultrasounds and 107 segmentations
Patient 22 has 238 ultrasounds and 238 segmentations
Patient 23 has 226 ultrasounds and 226 segmentations
Patient 24 has 162 ultrasounds and 162 segmentations
Patient 25 has 124 ultrasounds and 124 segmentations
Patient 26 has 209 ultrasounds and 209 segmentations
Patient 27 has 123 ultrasounds and 123 segmentations
Patient 28 has 181 ultrasounds and 181 segmentations
Patient 29 has 134 ultrasounds and 134 segmentations
Patient 30 has 145 ultrasounds and 145 segmentations
Patient 31 has 133 ultrasounds and 133 segmentations
Patient 32 has 179 ultrasounds and 179 segmentations

# Prepare validation rounds

if np.max(np.max(validation_schedule_patient)) > (n_patients - 1):
    raise Exception("Patient ID cannot be greater than {}".format(n_patients - 1))

num_validation_rounds = len(validation_schedule_patient)
print("Planning {} rounds of validation".format(num_validation_rounds))
for i in range(num_validation_rounds):
    print("Validation on patients {} in round {}".format(validation_schedule_patient[i], i))

Planning 5 rounds of validation
Validation on patients [0 1 2 3] in round 0
Validation on patients [ 7 11 13 18] in round 1
Validation on patients [ 9 14 21 31] in round 2
Validation on patients [19 22 29 32] in round 3
Validation on patients [20 23 27 30] in round 4

# Print training parameters, to archive them together with the notebook output.

time_sequence_start = datetime.datetime.now()

print("Timestamp for saved files: {}".format(save_timestamp))
print("\nTraining parameters")
print("Number of epochs:    {}".format(num_epochs))
print("Step size maximum:   {}".format(max_learning_rate))
print("Step size decay:     {}".format(learning_rate_decay))
print("Batch size:          {}".format(batch_size))
print("Regularization rate: {}".format(regularization_rate))
print("")
print("Saving validation predictions in: {}".format(val_data_fullpath))
print("Saving models in:                 {}".format(models_save_fullpath))

# ROC data will be saved in these containers

val_best_metrics    = dict()
val_fuzzy_metrics   = dict()
val_aurocs          = np.zeros(num_validation_rounds)
val_best_thresholds = np.zeros(num_validation_rounds)

# Perform validation rounds

for val_round_index in range(num_validation_rounds):
    
    # Prepare data arrays
    
    train_ultrasound_data = np.zeros(
        [0,
         ultrasound_arrays_by_patients[0].shape[1],
         ultrasound_arrays_by_patients[0].shape[2],
         ultrasound_arrays_by_patients[0].shape[3]])
    
    train_segmentation_data = np.zeros(
        [0,
         segmentation_arrays_by_patients[0].shape[1],
         segmentation_arrays_by_patients[0].shape[2],
         segmentation_arrays_by_patients[0].shape[3]])
    
    val_ultrasound_data = np.zeros(
        [0,
         ultrasound_arrays_by_patients[0].shape[1],
         ultrasound_arrays_by_patients[0].shape[2],
         ultrasound_arrays_by_patients[0].shape[3]])
    
    val_segmentation_data = np.zeros(
        [0,
         segmentation_arrays_by_patients[0].shape[1],
         segmentation_arrays_by_patients[0].shape[2],
         segmentation_arrays_by_patients[0].shape[3]])
    
    for patient_index in range(n_patients):
        if patient_index not in validation_schedule_patient[val_round_index]:
            train_ultrasound_data = np.concatenate((train_ultrasound_data,
                                                    ultrasound_arrays_by_patients[patient_index]))
            train_segmentation_data = np.concatenate((train_segmentation_data,
                                                      segmentation_arrays_by_patients[patient_index]))
        else:
            val_ultrasound_data = np.concatenate((val_ultrasound_data,
                                                 ultrasound_arrays_by_patients[patient_index]))
            val_segmentation_data = np.concatenate((val_segmentation_data,
                                                   segmentation_arrays_by_patients[patient_index]))
    
    n_train = train_ultrasound_data.shape[0]
    n_val = val_ultrasound_data.shape[0]
    
    print("\n*** Leave-one-out round # {}".format(val_round_index))
    print("    Training on {} images, validating on {} images...".format(n_train, n_val))
    
    val_segmentation_data_onehot = tf.keras.utils.to_categorical(val_segmentation_data, num_classes)
    
    # Create and train model
    
    model = unet.segmentation_unet(ultrasound_size, num_classes, filter_multiplier, regularization_rate)
    
    model.compile(
        optimizer=tf.keras.optimizers.Adam(lr=max_learning_rate, decay=learning_rate_decay),
        loss=unet.weighted_categorical_crossentropy(class_weights),
        metrics=["accuracy"]
    )
    """
    #----------------------------------------------------------------------------
    import os
    import sys
    import datetime
    from random import sample
    from pathlib import Path
    import girder_client
    import matplotlib.pyplot as plt
    import pandas as pd
    import tensorflow as tf
    import numpy as np

    from Spine.ultrasound_batch_generator import train_preprocess, train_preprocess_with_maps, generate_weight_maps
    from Spine import evaluation_metrics

    from Spine.models import (
        new_unet,
        weighted_categorical_crossentropy,
        weighted_categorical_crossentropy_with_maps,
    )
    import utils


    batch_size=128, 
    num_epochs=100, 
    sagittal_only=False, 
    num_frames=1, 
    with_maps=False, 
    learning_rate=0.002,
    lr_decay=False,
    dropout=0.0,
    use_attention=True,
    num_layers=5,
    filters=16,
    use_batch_norm=True,
    load_from_save=False,

    ultrasound_size = 128
    batch_size = 128
    num_classes = 2
    min_learning_rate = 0.00001
    class_weights = np.array([0.1, 0.9])
    learning_rate_decay = (0.002-0.00001) / 100
    num_epochs = 100

    model = new_unet(
                input_size = ultrasound_size,
                num_classes=num_classes,
                num_channels=num_frames,
                use_batch_norm=use_batch_norm,
                upsample_mode="deconv",  # 'deconv' or 'simple'
                dropout=dropout,
                dropout_type="spatial",
                use_attention=use_attention,
                filters=filters,
                num_layers=num_layers,
                output_activation="softmax",
            )

    learning_rate = 0.002
    loss_func = weighted_categorical_crossentropy(class_weights)
    
    preprocess_func = train_preprocess
    print(learning_rate, learning_rate_decay)
    model.compile(
                optimizer=tf.keras.optimizers.Adam(
                    lr=learning_rate, decay=learning_rate_decay
                ),
                loss=loss_func,
                metrics=["accuracy", evaluation_metrics.jaccard_coef, evaluation_metrics.dice_coef],
            )

#-----------------------------------------------------
    """
    
    model.summary() #prints the structure of the neural network. If you change the unet,
    #print this so yuo can see it.

    training_generator = generator.UltrasoundSegmentationBatchGenerator(
        train_ultrasound_data,
        train_segmentation_data[:, :, :, 0],
        batch_size,
        (ultrasound_size, ultrasound_size),
        max_shift_factor=max_shift_factor,
        min_zoom_factor=min_zoom_factor,
        max_zoom_factor=max_zoom_factor,
        max_rotation_angle=max_rotation_angle
    )
        
    training_time_start = datetime.datetime.now()
    print("TRAINING LOG",training_generator)
    if n_val > 0:
        training_log = model.fit_generator(
            training_generator,
            validation_data=(val_ultrasound_data, val_segmentation_data_onehot),
            epochs=num_epochs,
            verbose=0)
    else:
        training_log = model.fit_generator(training_generator, epochs=num_epochs, verbose=0)
    
    training_time_stop = datetime.datetime.now()
    
    # Pring training log
    
    print("  Training time: {}".format(training_time_stop-training_time_start))
    
    # Plot training loss and metrics
    
    fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))
    
    axes[0].plot(training_log.history['loss'], 'bo--')
    if n_val > 0:
        axes[0].plot(training_log.history['val_loss'], 'ro-')
    axes[0].set(xlabel='Epochs (n)', ylabel='Loss')
    if n_val > 0:
        axes[0].legend(['Training loss', 'Validation loss'])
    
    axes[1].plot(training_log.history['accuracy'], 'bo--')
    if n_val > 0:
        axes[1].plot(training_log.history['val_accuracy'], 'ro-')
    axes[1].set(xlabel='Epochs (n)', ylabel='Accuracy')
    if n_val > 0:
        axes[1].legend(['Training accuracy', 'Validation accuracy'])
    
    fig.tight_layout()
    
    # Archive trained model with unique filename based on notebook name and timestamp
    model_file_name = this_notebook_name + "_model-" + str(val_round_index) + "_" + save_timestamp + ".h5"
    model_fullname = os.path.join(models_save_fullpath, model_file_name)
    model.save(model_fullname)

    # Predict on validation data
    
    if n_val > 0:
        print(val_ultrasound_data.shape)
        y_pred_val  = model.predict(val_ultrasound_data)

        # Saving predictions for further evaluation

        val_prediction_filename = save_timestamp + "_prediction_" + str(val_round_index) + ".npy"
        val_prediction_fullname = os.path.join(val_data_fullpath, val_prediction_filename)
        np.save(val_prediction_fullname, y_pred_val)
        
        # Validation results

        vali_metrics_dicts, vali_best_threshold_index, vali_area = evaluation_metrics.compute_roc(
            roc_thresholds, y_pred_val, val_segmentation_data, acceptable_margin_mm, mm_per_pixel)

        val_fuzzy_metrics[val_round_index] = evaluation_metrics.compute_evaluation_metrics(
            y_pred_val, val_segmentation_data, acceptable_margin_mm, mm_per_pixel)

        val_best_metrics[val_round_index]    = vali_metrics_dicts[vali_best_threshold_index]
        val_aurocs[val_round_index]          = vali_area
        val_best_thresholds[val_round_index] = roc_thresholds[vali_best_threshold_index]
    
    # Printing total time of this validation round
    
    print("\nTotal round time:  {}".format(datetime.datetime.now() - training_time_start))
    print("")


time_sequence_stop = datetime.datetime.now()

print("\nTotal training time:   {}".format(time_sequence_stop - time_sequence_start))

Timestamp for saved files: 2020-11-25_15-52-21

Training parameters
Number of epochs:    200
Step size maximum:   0.02
Step size decay:     9.9995e-05
Batch size:          64
Regularization rate: 0.0001

Saving validation predictions in: /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\PredictionsValidation
Saving models in:                 /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\SavedModels

*** Leave-one-out round # 0
    Training on 5803 images, validating on 1515 images...
Model: "functional_1"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d (Conv2D)                 (None, 64, 64, 17)   170         input_1[0][0]                    
__________________________________________________________________________________________________
max_pooling2d (MaxPooling2D)    (None, 64, 64, 17)   0           conv2d[0][0]                     
__________________________________________________________________________________________________
conv2d_1 (Conv2D)               (None, 32, 32, 32)   4928        max_pooling2d[0][0]              
__________________________________________________________________________________________________
max_pooling2d_1 (MaxPooling2D)  (None, 32, 32, 32)   0           conv2d_1[0][0]                   
__________________________________________________________________________________________________
conv2d_2 (Conv2D)               (None, 16, 16, 47)   13583       max_pooling2d_1[0][0]            
__________________________________________________________________________________________________
max_pooling2d_2 (MaxPooling2D)  (None, 16, 16, 47)   0           conv2d_2[0][0]                   
__________________________________________________________________________________________________
conv2d_3 (Conv2D)               (None, 8, 8, 62)     26288       max_pooling2d_2[0][0]            
__________________________________________________________________________________________________
max_pooling2d_3 (MaxPooling2D)  (None, 8, 8, 62)     0           conv2d_3[0][0]                   
__________________________________________________________________________________________________
conv2d_4 (Conv2D)               (None, 4, 4, 77)     43043       max_pooling2d_3[0][0]            
__________________________________________________________________________________________________
max_pooling2d_4 (MaxPooling2D)  (None, 4, 4, 77)     0           conv2d_4[0][0]                   
__________________________________________________________________________________________________
conv2d_5 (Conv2D)               (None, 2, 2, 92)     63848       max_pooling2d_4[0][0]            
__________________________________________________________________________________________________
max_pooling2d_5 (MaxPooling2D)  (None, 2, 2, 92)     0           conv2d_5[0][0]                   
__________________________________________________________________________________________________
conv2d_6 (Conv2D)               (None, 1, 1, 107)    88703       max_pooling2d_5[0][0]            
__________________________________________________________________________________________________
max_pooling2d_6 (MaxPooling2D)  (None, 1, 1, 107)    0           conv2d_6[0][0]                   
__________________________________________________________________________________________________
up_sampling2d (UpSampling2D)    (None, 2, 2, 107)    0           max_pooling2d_6[0][0]            
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 2, 2, 199)    0           up_sampling2d[0][0]              
                                                                 max_pooling2d_5[0][0]            
__________________________________________________________________________________________________
conv2d_7 (Conv2D)               (None, 2, 2, 92)     293020      concatenate[0][0]                
__________________________________________________________________________________________________
batch_normalization (BatchNorma (None, 2, 2, 92)     368         conv2d_7[0][0]                   
__________________________________________________________________________________________________
up_sampling2d_1 (UpSampling2D)  (None, 4, 4, 92)     0           batch_normalization[0][0]        
__________________________________________________________________________________________________
concatenate_1 (Concatenate)     (None, 4, 4, 169)    0           up_sampling2d_1[0][0]            
                                                                 max_pooling2d_4[0][0]            
__________________________________________________________________________________________________
conv2d_8 (Conv2D)               (None, 4, 4, 77)     208285      concatenate_1[0][0]              
__________________________________________________________________________________________________
batch_normalization_1 (BatchNor (None, 4, 4, 77)     308         conv2d_8[0][0]                   
__________________________________________________________________________________________________
up_sampling2d_2 (UpSampling2D)  (None, 8, 8, 77)     0           batch_normalization_1[0][0]      
__________________________________________________________________________________________________
concatenate_2 (Concatenate)     (None, 8, 8, 139)    0           up_sampling2d_2[0][0]            
                                                                 max_pooling2d_3[0][0]            
__________________________________________________________________________________________________
conv2d_9 (Conv2D)               (None, 8, 8, 62)     137950      concatenate_2[0][0]              
__________________________________________________________________________________________________
batch_normalization_2 (BatchNor (None, 8, 8, 62)     248         conv2d_9[0][0]                   
__________________________________________________________________________________________________
up_sampling2d_3 (UpSampling2D)  (None, 16, 16, 62)   0           batch_normalization_2[0][0]      
__________________________________________________________________________________________________
concatenate_3 (Concatenate)     (None, 16, 16, 109)  0           up_sampling2d_3[0][0]            
                                                                 max_pooling2d_2[0][0]            
__________________________________________________________________________________________________
conv2d_10 (Conv2D)              (None, 16, 16, 47)   82015       concatenate_3[0][0]              
__________________________________________________________________________________________________
batch_normalization_3 (BatchNor (None, 16, 16, 47)   188         conv2d_10[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_4 (UpSampling2D)  (None, 32, 32, 47)   0           batch_normalization_3[0][0]      
__________________________________________________________________________________________________
concatenate_4 (Concatenate)     (None, 32, 32, 79)   0           up_sampling2d_4[0][0]            
                                                                 max_pooling2d_1[0][0]            
__________________________________________________________________________________________________
conv2d_11 (Conv2D)              (None, 32, 32, 32)   40480       concatenate_4[0][0]              
__________________________________________________________________________________________________
batch_normalization_4 (BatchNor (None, 32, 32, 32)   128         conv2d_11[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_5 (UpSampling2D)  (None, 64, 64, 32)   0           batch_normalization_4[0][0]      
__________________________________________________________________________________________________
concatenate_5 (Concatenate)     (None, 64, 64, 49)   0           up_sampling2d_5[0][0]            
                                                                 max_pooling2d[0][0]              
__________________________________________________________________________________________________
conv2d_12 (Conv2D)              (None, 64, 64, 17)   13345       concatenate_5[0][0]              
__________________________________________________________________________________________________
batch_normalization_5 (BatchNor (None, 64, 64, 17)   68          conv2d_12[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_6 (UpSampling2D)  (None, 128, 128, 17) 0           batch_normalization_5[0][0]      
__________________________________________________________________________________________________
concatenate_6 (Concatenate)     (None, 128, 128, 18) 0           up_sampling2d_6[0][0]            
                                                                 input_1[0][0]                    
__________________________________________________________________________________________________
conv2d_13 (Conv2D)              (None, 128, 128, 2)  578         concatenate_6[0][0]              
==================================================================================================
Total params: 1,017,544
Trainable params: 1,016,890
Non-trainable params: 654
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000002AC2B6C5370>
WARNING:tensorflow:From <ipython-input-7-639783e879e1>:175: Model.fit_generator (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.
Instructions for updating:
Please use Model.fit, which supports generators.
  Training time: 6:42:10.418664
(1515, 128, 128, 1)

Total round time:  6:42:23.878649


*** Leave-one-out round # 1
    Training on 6290 images, validating on 1028 images...
Model: "functional_3"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_2 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_14 (Conv2D)              (None, 64, 64, 17)   170         input_2[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_7 (MaxPooling2D)  (None, 64, 64, 17)   0           conv2d_14[0][0]                  
__________________________________________________________________________________________________
conv2d_15 (Conv2D)              (None, 32, 32, 32)   4928        max_pooling2d_7[0][0]            
__________________________________________________________________________________________________
max_pooling2d_8 (MaxPooling2D)  (None, 32, 32, 32)   0           conv2d_15[0][0]                  
__________________________________________________________________________________________________
conv2d_16 (Conv2D)              (None, 16, 16, 47)   13583       max_pooling2d_8[0][0]            
__________________________________________________________________________________________________
max_pooling2d_9 (MaxPooling2D)  (None, 16, 16, 47)   0           conv2d_16[0][0]                  
__________________________________________________________________________________________________
conv2d_17 (Conv2D)              (None, 8, 8, 62)     26288       max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
max_pooling2d_10 (MaxPooling2D) (None, 8, 8, 62)     0           conv2d_17[0][0]                  
__________________________________________________________________________________________________
conv2d_18 (Conv2D)              (None, 4, 4, 77)     43043       max_pooling2d_10[0][0]           
__________________________________________________________________________________________________
max_pooling2d_11 (MaxPooling2D) (None, 4, 4, 77)     0           conv2d_18[0][0]                  
__________________________________________________________________________________________________
conv2d_19 (Conv2D)              (None, 2, 2, 92)     63848       max_pooling2d_11[0][0]           
__________________________________________________________________________________________________
max_pooling2d_12 (MaxPooling2D) (None, 2, 2, 92)     0           conv2d_19[0][0]                  
__________________________________________________________________________________________________
conv2d_20 (Conv2D)              (None, 1, 1, 107)    88703       max_pooling2d_12[0][0]           
__________________________________________________________________________________________________
max_pooling2d_13 (MaxPooling2D) (None, 1, 1, 107)    0           conv2d_20[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_7 (UpSampling2D)  (None, 2, 2, 107)    0           max_pooling2d_13[0][0]           
__________________________________________________________________________________________________
concatenate_7 (Concatenate)     (None, 2, 2, 199)    0           up_sampling2d_7[0][0]            
                                                                 max_pooling2d_12[0][0]           
__________________________________________________________________________________________________
conv2d_21 (Conv2D)              (None, 2, 2, 92)     293020      concatenate_7[0][0]              
__________________________________________________________________________________________________
batch_normalization_6 (BatchNor (None, 2, 2, 92)     368         conv2d_21[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_8 (UpSampling2D)  (None, 4, 4, 92)     0           batch_normalization_6[0][0]      
__________________________________________________________________________________________________
concatenate_8 (Concatenate)     (None, 4, 4, 169)    0           up_sampling2d_8[0][0]            
                                                                 max_pooling2d_11[0][0]           
__________________________________________________________________________________________________
conv2d_22 (Conv2D)              (None, 4, 4, 77)     208285      concatenate_8[0][0]              
__________________________________________________________________________________________________
batch_normalization_7 (BatchNor (None, 4, 4, 77)     308         conv2d_22[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_9 (UpSampling2D)  (None, 8, 8, 77)     0           batch_normalization_7[0][0]      
__________________________________________________________________________________________________
concatenate_9 (Concatenate)     (None, 8, 8, 139)    0           up_sampling2d_9[0][0]            
                                                                 max_pooling2d_10[0][0]           
__________________________________________________________________________________________________
conv2d_23 (Conv2D)              (None, 8, 8, 62)     137950      concatenate_9[0][0]              
__________________________________________________________________________________________________
batch_normalization_8 (BatchNor (None, 8, 8, 62)     248         conv2d_23[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_10 (UpSampling2D) (None, 16, 16, 62)   0           batch_normalization_8[0][0]      
__________________________________________________________________________________________________
concatenate_10 (Concatenate)    (None, 16, 16, 109)  0           up_sampling2d_10[0][0]           
                                                                 max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
conv2d_24 (Conv2D)              (None, 16, 16, 47)   82015       concatenate_10[0][0]             
__________________________________________________________________________________________________
batch_normalization_9 (BatchNor (None, 16, 16, 47)   188         conv2d_24[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_11 (UpSampling2D) (None, 32, 32, 47)   0           batch_normalization_9[0][0]      
__________________________________________________________________________________________________
concatenate_11 (Concatenate)    (None, 32, 32, 79)   0           up_sampling2d_11[0][0]           
                                                                 max_pooling2d_8[0][0]            
__________________________________________________________________________________________________
conv2d_25 (Conv2D)              (None, 32, 32, 32)   40480       concatenate_11[0][0]             
__________________________________________________________________________________________________
batch_normalization_10 (BatchNo (None, 32, 32, 32)   128         conv2d_25[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_12 (UpSampling2D) (None, 64, 64, 32)   0           batch_normalization_10[0][0]     
__________________________________________________________________________________________________
concatenate_12 (Concatenate)    (None, 64, 64, 49)   0           up_sampling2d_12[0][0]           
                                                                 max_pooling2d_7[0][0]            
__________________________________________________________________________________________________
conv2d_26 (Conv2D)              (None, 64, 64, 17)   13345       concatenate_12[0][0]             
__________________________________________________________________________________________________
batch_normalization_11 (BatchNo (None, 64, 64, 17)   68          conv2d_26[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_13 (UpSampling2D) (None, 128, 128, 17) 0           batch_normalization_11[0][0]     
__________________________________________________________________________________________________
concatenate_13 (Concatenate)    (None, 128, 128, 18) 0           up_sampling2d_13[0][0]           
                                                                 input_2[0][0]                    
__________________________________________________________________________________________________
conv2d_27 (Conv2D)              (None, 128, 128, 2)  578         concatenate_13[0][0]             
==================================================================================================
Total params: 1,017,544
Trainable params: 1,016,890
Non-trainable params: 654
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000002AC2DC35AC0>
  Training time: 7:10:14.589074
(1028, 128, 128, 1)

Total round time:  7:10:23.704911


*** Leave-one-out round # 2
    Training on 6670 images, validating on 648 images...
Model: "functional_5"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_3 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_28 (Conv2D)              (None, 64, 64, 17)   170         input_3[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_14 (MaxPooling2D) (None, 64, 64, 17)   0           conv2d_28[0][0]                  
__________________________________________________________________________________________________
conv2d_29 (Conv2D)              (None, 32, 32, 32)   4928        max_pooling2d_14[0][0]           
__________________________________________________________________________________________________
max_pooling2d_15 (MaxPooling2D) (None, 32, 32, 32)   0           conv2d_29[0][0]                  
__________________________________________________________________________________________________
conv2d_30 (Conv2D)              (None, 16, 16, 47)   13583       max_pooling2d_15[0][0]           
__________________________________________________________________________________________________
max_pooling2d_16 (MaxPooling2D) (None, 16, 16, 47)   0           conv2d_30[0][0]                  
__________________________________________________________________________________________________
conv2d_31 (Conv2D)              (None, 8, 8, 62)     26288       max_pooling2d_16[0][0]           
__________________________________________________________________________________________________
max_pooling2d_17 (MaxPooling2D) (None, 8, 8, 62)     0           conv2d_31[0][0]                  
__________________________________________________________________________________________________
conv2d_32 (Conv2D)              (None, 4, 4, 77)     43043       max_pooling2d_17[0][0]           
__________________________________________________________________________________________________
max_pooling2d_18 (MaxPooling2D) (None, 4, 4, 77)     0           conv2d_32[0][0]                  
__________________________________________________________________________________________________
conv2d_33 (Conv2D)              (None, 2, 2, 92)     63848       max_pooling2d_18[0][0]           
__________________________________________________________________________________________________
max_pooling2d_19 (MaxPooling2D) (None, 2, 2, 92)     0           conv2d_33[0][0]                  
__________________________________________________________________________________________________
conv2d_34 (Conv2D)              (None, 1, 1, 107)    88703       max_pooling2d_19[0][0]           
__________________________________________________________________________________________________
max_pooling2d_20 (MaxPooling2D) (None, 1, 1, 107)    0           conv2d_34[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_14 (UpSampling2D) (None, 2, 2, 107)    0           max_pooling2d_20[0][0]           
__________________________________________________________________________________________________
concatenate_14 (Concatenate)    (None, 2, 2, 199)    0           up_sampling2d_14[0][0]           
                                                                 max_pooling2d_19[0][0]           
__________________________________________________________________________________________________
conv2d_35 (Conv2D)              (None, 2, 2, 92)     293020      concatenate_14[0][0]             
__________________________________________________________________________________________________
batch_normalization_12 (BatchNo (None, 2, 2, 92)     368         conv2d_35[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_15 (UpSampling2D) (None, 4, 4, 92)     0           batch_normalization_12[0][0]     
__________________________________________________________________________________________________
concatenate_15 (Concatenate)    (None, 4, 4, 169)    0           up_sampling2d_15[0][0]           
                                                                 max_pooling2d_18[0][0]           
__________________________________________________________________________________________________
conv2d_36 (Conv2D)              (None, 4, 4, 77)     208285      concatenate_15[0][0]             
__________________________________________________________________________________________________
batch_normalization_13 (BatchNo (None, 4, 4, 77)     308         conv2d_36[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_16 (UpSampling2D) (None, 8, 8, 77)     0           batch_normalization_13[0][0]     
__________________________________________________________________________________________________
concatenate_16 (Concatenate)    (None, 8, 8, 139)    0           up_sampling2d_16[0][0]           
                                                                 max_pooling2d_17[0][0]           
__________________________________________________________________________________________________
conv2d_37 (Conv2D)              (None, 8, 8, 62)     137950      concatenate_16[0][0]             
__________________________________________________________________________________________________
batch_normalization_14 (BatchNo (None, 8, 8, 62)     248         conv2d_37[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_17 (UpSampling2D) (None, 16, 16, 62)   0           batch_normalization_14[0][0]     
__________________________________________________________________________________________________
concatenate_17 (Concatenate)    (None, 16, 16, 109)  0           up_sampling2d_17[0][0]           
                                                                 max_pooling2d_16[0][0]           
__________________________________________________________________________________________________
conv2d_38 (Conv2D)              (None, 16, 16, 47)   82015       concatenate_17[0][0]             
__________________________________________________________________________________________________
batch_normalization_15 (BatchNo (None, 16, 16, 47)   188         conv2d_38[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_18 (UpSampling2D) (None, 32, 32, 47)   0           batch_normalization_15[0][0]     
__________________________________________________________________________________________________
concatenate_18 (Concatenate)    (None, 32, 32, 79)   0           up_sampling2d_18[0][0]           
                                                                 max_pooling2d_15[0][0]           
__________________________________________________________________________________________________
conv2d_39 (Conv2D)              (None, 32, 32, 32)   40480       concatenate_18[0][0]             
__________________________________________________________________________________________________
batch_normalization_16 (BatchNo (None, 32, 32, 32)   128         conv2d_39[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_19 (UpSampling2D) (None, 64, 64, 32)   0           batch_normalization_16[0][0]     
__________________________________________________________________________________________________
concatenate_19 (Concatenate)    (None, 64, 64, 49)   0           up_sampling2d_19[0][0]           
                                                                 max_pooling2d_14[0][0]           
__________________________________________________________________________________________________
conv2d_40 (Conv2D)              (None, 64, 64, 17)   13345       concatenate_19[0][0]             
__________________________________________________________________________________________________
batch_normalization_17 (BatchNo (None, 64, 64, 17)   68          conv2d_40[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_20 (UpSampling2D) (None, 128, 128, 17) 0           batch_normalization_17[0][0]     
__________________________________________________________________________________________________
concatenate_20 (Concatenate)    (None, 128, 128, 18) 0           up_sampling2d_20[0][0]           
                                                                 input_3[0][0]                    
__________________________________________________________________________________________________
conv2d_41 (Conv2D)              (None, 128, 128, 2)  578         concatenate_20[0][0]             
==================================================================================================
Total params: 1,017,544
Trainable params: 1,016,890
Non-trainable params: 654
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000002AC34315F70>
  Training time: 7:34:28.907510
(648, 128, 128, 1)

Total round time:  7:34:34.866427


*** Leave-one-out round # 3
    Training on 6580 images, validating on 738 images...
Model: "functional_7"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_4 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_42 (Conv2D)              (None, 64, 64, 17)   170         input_4[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_21 (MaxPooling2D) (None, 64, 64, 17)   0           conv2d_42[0][0]                  
__________________________________________________________________________________________________
conv2d_43 (Conv2D)              (None, 32, 32, 32)   4928        max_pooling2d_21[0][0]           
__________________________________________________________________________________________________
max_pooling2d_22 (MaxPooling2D) (None, 32, 32, 32)   0           conv2d_43[0][0]                  
__________________________________________________________________________________________________
conv2d_44 (Conv2D)              (None, 16, 16, 47)   13583       max_pooling2d_22[0][0]           
__________________________________________________________________________________________________
max_pooling2d_23 (MaxPooling2D) (None, 16, 16, 47)   0           conv2d_44[0][0]                  
__________________________________________________________________________________________________
conv2d_45 (Conv2D)              (None, 8, 8, 62)     26288       max_pooling2d_23[0][0]           
__________________________________________________________________________________________________
max_pooling2d_24 (MaxPooling2D) (None, 8, 8, 62)     0           conv2d_45[0][0]                  
__________________________________________________________________________________________________
conv2d_46 (Conv2D)              (None, 4, 4, 77)     43043       max_pooling2d_24[0][0]           
__________________________________________________________________________________________________
max_pooling2d_25 (MaxPooling2D) (None, 4, 4, 77)     0           conv2d_46[0][0]                  
__________________________________________________________________________________________________
conv2d_47 (Conv2D)              (None, 2, 2, 92)     63848       max_pooling2d_25[0][0]           
__________________________________________________________________________________________________
max_pooling2d_26 (MaxPooling2D) (None, 2, 2, 92)     0           conv2d_47[0][0]                  
__________________________________________________________________________________________________
conv2d_48 (Conv2D)              (None, 1, 1, 107)    88703       max_pooling2d_26[0][0]           
__________________________________________________________________________________________________
max_pooling2d_27 (MaxPooling2D) (None, 1, 1, 107)    0           conv2d_48[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_21 (UpSampling2D) (None, 2, 2, 107)    0           max_pooling2d_27[0][0]           
__________________________________________________________________________________________________
concatenate_21 (Concatenate)    (None, 2, 2, 199)    0           up_sampling2d_21[0][0]           
                                                                 max_pooling2d_26[0][0]           
__________________________________________________________________________________________________
conv2d_49 (Conv2D)              (None, 2, 2, 92)     293020      concatenate_21[0][0]             
__________________________________________________________________________________________________
batch_normalization_18 (BatchNo (None, 2, 2, 92)     368         conv2d_49[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_22 (UpSampling2D) (None, 4, 4, 92)     0           batch_normalization_18[0][0]     
__________________________________________________________________________________________________
concatenate_22 (Concatenate)    (None, 4, 4, 169)    0           up_sampling2d_22[0][0]           
                                                                 max_pooling2d_25[0][0]           
__________________________________________________________________________________________________
conv2d_50 (Conv2D)              (None, 4, 4, 77)     208285      concatenate_22[0][0]             
__________________________________________________________________________________________________
batch_normalization_19 (BatchNo (None, 4, 4, 77)     308         conv2d_50[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_23 (UpSampling2D) (None, 8, 8, 77)     0           batch_normalization_19[0][0]     
__________________________________________________________________________________________________
concatenate_23 (Concatenate)    (None, 8, 8, 139)    0           up_sampling2d_23[0][0]           
                                                                 max_pooling2d_24[0][0]           
__________________________________________________________________________________________________
conv2d_51 (Conv2D)              (None, 8, 8, 62)     137950      concatenate_23[0][0]             
__________________________________________________________________________________________________
batch_normalization_20 (BatchNo (None, 8, 8, 62)     248         conv2d_51[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_24 (UpSampling2D) (None, 16, 16, 62)   0           batch_normalization_20[0][0]     
__________________________________________________________________________________________________
concatenate_24 (Concatenate)    (None, 16, 16, 109)  0           up_sampling2d_24[0][0]           
                                                                 max_pooling2d_23[0][0]           
__________________________________________________________________________________________________
conv2d_52 (Conv2D)              (None, 16, 16, 47)   82015       concatenate_24[0][0]             
__________________________________________________________________________________________________
batch_normalization_21 (BatchNo (None, 16, 16, 47)   188         conv2d_52[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_25 (UpSampling2D) (None, 32, 32, 47)   0           batch_normalization_21[0][0]     
__________________________________________________________________________________________________
concatenate_25 (Concatenate)    (None, 32, 32, 79)   0           up_sampling2d_25[0][0]           
                                                                 max_pooling2d_22[0][0]           
__________________________________________________________________________________________________
conv2d_53 (Conv2D)              (None, 32, 32, 32)   40480       concatenate_25[0][0]             
__________________________________________________________________________________________________
batch_normalization_22 (BatchNo (None, 32, 32, 32)   128         conv2d_53[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_26 (UpSampling2D) (None, 64, 64, 32)   0           batch_normalization_22[0][0]     
__________________________________________________________________________________________________
concatenate_26 (Concatenate)    (None, 64, 64, 49)   0           up_sampling2d_26[0][0]           
                                                                 max_pooling2d_21[0][0]           
__________________________________________________________________________________________________
conv2d_54 (Conv2D)              (None, 64, 64, 17)   13345       concatenate_26[0][0]             
__________________________________________________________________________________________________
batch_normalization_23 (BatchNo (None, 64, 64, 17)   68          conv2d_54[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_27 (UpSampling2D) (None, 128, 128, 17) 0           batch_normalization_23[0][0]     
__________________________________________________________________________________________________
concatenate_27 (Concatenate)    (None, 128, 128, 18) 0           up_sampling2d_27[0][0]           
                                                                 input_4[0][0]                    
__________________________________________________________________________________________________
conv2d_55 (Conv2D)              (None, 128, 128, 2)  578         concatenate_27[0][0]             
==================================================================================================
Total params: 1,017,544
Trainable params: 1,016,890
Non-trainable params: 654
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000002AC2D200400>
  Training time: 7:27:11.132603
(738, 128, 128, 1)

Total round time:  7:27:17.788498


*** Leave-one-out round # 4
    Training on 6689 images, validating on 629 images...
Model: "functional_9"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_5 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_56 (Conv2D)              (None, 64, 64, 17)   170         input_5[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_28 (MaxPooling2D) (None, 64, 64, 17)   0           conv2d_56[0][0]                  
__________________________________________________________________________________________________
conv2d_57 (Conv2D)              (None, 32, 32, 32)   4928        max_pooling2d_28[0][0]           
__________________________________________________________________________________________________
max_pooling2d_29 (MaxPooling2D) (None, 32, 32, 32)   0           conv2d_57[0][0]                  
__________________________________________________________________________________________________
conv2d_58 (Conv2D)              (None, 16, 16, 47)   13583       max_pooling2d_29[0][0]           
__________________________________________________________________________________________________
max_pooling2d_30 (MaxPooling2D) (None, 16, 16, 47)   0           conv2d_58[0][0]                  
__________________________________________________________________________________________________
conv2d_59 (Conv2D)              (None, 8, 8, 62)     26288       max_pooling2d_30[0][0]           
__________________________________________________________________________________________________
max_pooling2d_31 (MaxPooling2D) (None, 8, 8, 62)     0           conv2d_59[0][0]                  
__________________________________________________________________________________________________
conv2d_60 (Conv2D)              (None, 4, 4, 77)     43043       max_pooling2d_31[0][0]           
__________________________________________________________________________________________________
max_pooling2d_32 (MaxPooling2D) (None, 4, 4, 77)     0           conv2d_60[0][0]                  
__________________________________________________________________________________________________
conv2d_61 (Conv2D)              (None, 2, 2, 92)     63848       max_pooling2d_32[0][0]           
__________________________________________________________________________________________________
max_pooling2d_33 (MaxPooling2D) (None, 2, 2, 92)     0           conv2d_61[0][0]                  
__________________________________________________________________________________________________
conv2d_62 (Conv2D)              (None, 1, 1, 107)    88703       max_pooling2d_33[0][0]           
__________________________________________________________________________________________________
max_pooling2d_34 (MaxPooling2D) (None, 1, 1, 107)    0           conv2d_62[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_28 (UpSampling2D) (None, 2, 2, 107)    0           max_pooling2d_34[0][0]           
__________________________________________________________________________________________________
concatenate_28 (Concatenate)    (None, 2, 2, 199)    0           up_sampling2d_28[0][0]           
                                                                 max_pooling2d_33[0][0]           
__________________________________________________________________________________________________
conv2d_63 (Conv2D)              (None, 2, 2, 92)     293020      concatenate_28[0][0]             
__________________________________________________________________________________________________
batch_normalization_24 (BatchNo (None, 2, 2, 92)     368         conv2d_63[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_29 (UpSampling2D) (None, 4, 4, 92)     0           batch_normalization_24[0][0]     
__________________________________________________________________________________________________
concatenate_29 (Concatenate)    (None, 4, 4, 169)    0           up_sampling2d_29[0][0]           
                                                                 max_pooling2d_32[0][0]           
__________________________________________________________________________________________________
conv2d_64 (Conv2D)              (None, 4, 4, 77)     208285      concatenate_29[0][0]             
__________________________________________________________________________________________________
batch_normalization_25 (BatchNo (None, 4, 4, 77)     308         conv2d_64[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_30 (UpSampling2D) (None, 8, 8, 77)     0           batch_normalization_25[0][0]     
__________________________________________________________________________________________________
concatenate_30 (Concatenate)    (None, 8, 8, 139)    0           up_sampling2d_30[0][0]           
                                                                 max_pooling2d_31[0][0]           
__________________________________________________________________________________________________
conv2d_65 (Conv2D)              (None, 8, 8, 62)     137950      concatenate_30[0][0]             
__________________________________________________________________________________________________
batch_normalization_26 (BatchNo (None, 8, 8, 62)     248         conv2d_65[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_31 (UpSampling2D) (None, 16, 16, 62)   0           batch_normalization_26[0][0]     
__________________________________________________________________________________________________
concatenate_31 (Concatenate)    (None, 16, 16, 109)  0           up_sampling2d_31[0][0]           
                                                                 max_pooling2d_30[0][0]           
__________________________________________________________________________________________________
conv2d_66 (Conv2D)              (None, 16, 16, 47)   82015       concatenate_31[0][0]             
__________________________________________________________________________________________________
batch_normalization_27 (BatchNo (None, 16, 16, 47)   188         conv2d_66[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_32 (UpSampling2D) (None, 32, 32, 47)   0           batch_normalization_27[0][0]     
__________________________________________________________________________________________________
concatenate_32 (Concatenate)    (None, 32, 32, 79)   0           up_sampling2d_32[0][0]           
                                                                 max_pooling2d_29[0][0]           
__________________________________________________________________________________________________
conv2d_67 (Conv2D)              (None, 32, 32, 32)   40480       concatenate_32[0][0]             
__________________________________________________________________________________________________
batch_normalization_28 (BatchNo (None, 32, 32, 32)   128         conv2d_67[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_33 (UpSampling2D) (None, 64, 64, 32)   0           batch_normalization_28[0][0]     
__________________________________________________________________________________________________
concatenate_33 (Concatenate)    (None, 64, 64, 49)   0           up_sampling2d_33[0][0]           
                                                                 max_pooling2d_28[0][0]           
__________________________________________________________________________________________________
conv2d_68 (Conv2D)              (None, 64, 64, 17)   13345       concatenate_33[0][0]             
__________________________________________________________________________________________________
batch_normalization_29 (BatchNo (None, 64, 64, 17)   68          conv2d_68[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_34 (UpSampling2D) (None, 128, 128, 17) 0           batch_normalization_29[0][0]     
__________________________________________________________________________________________________
concatenate_34 (Concatenate)    (None, 128, 128, 18) 0           up_sampling2d_34[0][0]           
                                                                 input_5[0][0]                    
__________________________________________________________________________________________________
conv2d_69 (Conv2D)              (None, 128, 128, 2)  578         concatenate_34[0][0]             
==================================================================================================
Total params: 1,017,544
Trainable params: 1,016,890
Non-trainable params: 654
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000002AC2C9F2B20>
  Training time: 7:33:34.531170
(629, 128, 128, 1)

Total round time:  7:33:40.234246


Total training time:   1 day, 12:28:56.308730

# Arrange results in tables

metric_labels = [
    "AUROC",
    "best thresh",
    "best TP",
    "best FP",
    "best recall",
    "best precis",
    "fuzzy recall",
    "fuzzy precis",
    "fuzzy Fscore"
]

results_labels = []
  
for label in metric_labels:
    results_labels.append("Vali " + label)

results_df = pd.DataFrame(columns = results_labels)

for i in range(num_validation_rounds):
    if i in val_best_metrics.keys():
        results_df.loc[i] = [
            val_aurocs[i],
            val_best_thresholds[i],
            val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.FALSE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.RECALL],
            val_best_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.RECALL],
            val_fuzzy_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.FSCORE]
        ]

display(results_df)

print("\nAverages")

results_means_df = results_df.mean()
display(results_means_df)

	Vali AUROC	Vali best thresh	Vali best TP	Vali best FP	Vali best recall	Vali best precis	Vali fuzzy recall	Vali fuzzy precis	Vali fuzzy Fscore
0	0.986887	0.010	0.976789	0.031978	0.976789	0.204770	0.791711	0.480316	0.597898
1	0.781408	0.010	0.721994	0.111493	0.721994	0.076116	0.377772	0.135324	0.199267
2	0.947425	0.010	0.952871	0.098197	0.952871	0.176882	0.753027	0.312603	0.441801
3	0.786941	0.001	0.810828	0.106515	0.810828	0.186159	0.416392	0.475146	0.443833
4	0.739280	0.001	0.758853	0.088894	0.758853	0.232296	0.374956	0.559779	0.449096

Averages

Vali AUROC           0.848388
Vali best thresh     0.006400
Vali best TP         0.844267
Vali best FP         0.087415
Vali best recall     0.844267
Vali best precis     0.175244
Vali fuzzy recall    0.542772
Vali fuzzy precis    0.392634
Vali fuzzy Fscore    0.426379
dtype: float64

# Print the last ROC curve for visual verification that we catch the optimal point

n = len(roc_thresholds)

roc_x = np.zeros(n)
roc_y = np.zeros(n)

for i in range(n):
    roc_x[i] = vali_metrics_dicts[i][evaluation_metrics.FALSE_POSITIVE_RATE]
    roc_y[i] = vali_metrics_dicts[i][evaluation_metrics.SENSITIVITY]
    # print("Threshold = {0:4.2f}  False pos rate = {1:4.2f}  Sensitivity = {2:4.2f}"
    #       .format(roc_thresholds[i], roc_x[i], roc_y[i]))

    
plt.figure()
plt.ylim(-0.01, 1.01)
plt.xlim(-0.01, 1.01)
plt.plot(roc_x, roc_y, color='darkred', lw=2)

[<matplotlib.lines.Line2D at 0x2ac2d5b71c0>]

# Save results table

csv_filename = this_notebook_name + "_" + save_timestamp + ".csv"
csv_fullname = os.path.join(results_save_fullpath, csv_filename)
results_df.to_csv(csv_fullname)

print("Results saved to: {}".format(csv_fullname))

Results saved to: /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\SavedResults\BreastSegmentationStudy-TF2_2020-11-25_15-52-21.csv

# Display sample results

# determine the available ultrasounds in the last round of validation
last_round = validation_schedule_patient[-1]
print(val_ultrasound_data.shape)
prev_vali = 0

#20: [63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100]
#23: [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153]
#27: [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379]
#30: [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]
count = 0
patientNumber = [20, 23, 27, 30]
sample_indices = [[63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100], [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153], [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379], [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]]
for j in range(len(last_round)):
    
    num_vali =  prev_vali + ultrasound_arrays_by_patients[last_round[j]].shape[0]
    num_show = 20
    if num_vali < num_show:
        num_show = 0
    num_col = 4

    indices = [i for i in range(prev_vali, num_vali)]
    #sample_indices = sample(indices, num_show)
    print("Showing image slices for patient", patientNumber[count], ":", sample_indices[count])
    #sample_indices = [prev_vali]
    
    # update
    prev_vali = num_vali
    threshold = 0.5
    
    fig = plt.figure(figsize=(18, num_show*5))
    for i in range(num_show):
        a0 = fig.add_subplot(num_show, num_col, i*num_col+1)
        img0 = a0.imshow(np.flipud(val_ultrasound_data[sample_indices[count][i], :, :, 0].astype(np.float32)))
        a0.set_title("Patient #{} - Ultrasound #{}".format(last_round[j], sample_indices[count][i]))
        a1 = fig.add_subplot(num_show, num_col, i*num_col+2)
        img1 = a1.imshow(np.flipud(val_segmentation_data_onehot[sample_indices[count][i], :, :, 1]), vmin=0.0, vmax=1.0)
        a1.set_title("Segmentation #{}".format(sample_indices[count][i]))
        c = fig.colorbar(img1, fraction=0.046, pad=0.04)
        a2 = fig.add_subplot(num_show, num_col, i*num_col+3)
        img2 = a2.imshow(np.flipud(y_pred_val[sample_indices[count][i], :, :, 1]), vmin=0.0, vmax=1.0)
        a2.set_title("Prediction #{}".format(sample_indices[count][i]))
        c = fig.colorbar(img2, fraction=0.046, pad=0.04)
        a3 = fig.add_subplot(num_show, num_col, i*num_col+4)
        img3 = a3.imshow((np.flipud(y_pred_val[sample_indices[count][i], :, :, 1]) > threshold), vmin=0.0, vmax=1.0)
        c = fig.colorbar(img3, fraction=0.046, pad=0.04)
        a3.set_title("Thresholded #{}".format(sample_indices[count][i]))
    count = count + 1

(629, 128, 128, 1)
Showing image slices for patient 20 : [63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100]
Showing image slices for patient 23 : [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153]
Showing image slices for patient 27 : [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379]
Showing image slices for patient 30 : [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]

# Save notebook so all output is archived by the next cell

from IPython.display import Javascript
script = '''
require(["base/js/namespace"],function(Jupyter) {
    Jupyter.notebook.save_checkpoint();
});
'''
Javascript(script)

# Export HTML copy of this notebook

notebook_file_name =  "/Users/Josh Ehrlich/Courses/CISC881/Project/data/"+this_notebook_name + "_" + save_timestamp + ".html"
notebook_fullname = os.path.join(notebooks_save_fullpath, notebook_file_name)

os.system("jupyter nbconvert --to html " + this_notebook_name + " --output " + notebook_fullname)
print("Notebook saved to: {}".format(notebook_fullname))

notebook_file_name

print("Notebook_name", this_notebook_name)
print("fullname", notebook_fullname)